const superagent = require('superagent')
const cheerio = require('cheerio')
const async = require('async')
const querystring = require('querystring')
const crypto = require('crypto')
const mysql = require('mysql')
const connection = mysql.createConnection({
    host: '10.19.103.71',
    port: '3306',
    user: 'root',
    password: 'Duo123uclo',
    database: 'hzkb'
})
connection.connect()
const get_jd_promotion_link = require('./get_jd_promotion_link.js')

let tb_pid = 'mm_114783873_25036753_85938290'
let continue_spider = true

//从APP的API获取数据
let spider_data = function(){
    let num_to_spider = 3
    let current_num = 0
    let since_str = ''
    let base_spider_url = 'https://guangdiu.com/api/getlist.php?count=30&v=21'
    async.whilst(
        function(){
            current_num++
            if(continue_spider){
                return current_num <= num_to_spider
            }else{
                return false
            }
        },
        function(whilst_cb){
            superagent.get(base_spider_url + since_str)
            .set('Host', 'guangdiu.com')//important
            .set('User-Agent', 'GDeals/2.3 (iPhone; iOS 11.2; Scale/3.00)')
            .set('Accept', '*/*')
            .end(function(err, res){
                let items = JSON.parse(res.text).data
                console.log(items.length)
                since_str = '&sinceid=' + items[items.length - 1].id
                data_wash_and_store_data(items, whilst_cb)
            })
        },
        function(err, result){
            console.log(new Date().toLocaleString() + '抓取——清洗——存储完毕')
            spider_hot_list()
        }
    )
}
exports.spider_data = spider_data

function data_wash_and_store_data(items, whilst_cb){
    let i = -1
    async.whilst(
        function(){
            i++
            return i < items.length
        },
        function(cb){
            //目前只处理淘宝和京东的商品
            if(items[i].mall != '天猫' && items[i].mall != '京东商城'){
                cb(null, 'success')
                return
            }
            let product_info = {
                guangdiuid: items[i].id,
                title: items[i].title,
                pubtime: new Date(new Date(items[i].pubtime).getTime() + 1000 * 60 * 30).getTime(),//半小时抓一次数据，抓到的数据后延半小时更新
                image: items[i].image.match(new RegExp(/(\S+)\?imageView\S+/))[1],
                fromsite: items[i].fromsite,
                mall: items[i].mall
            }
            get_detail_info(items[i], product_info, cb)
        },
        function(err, result){
            whilst_cb(null, 'success')
        }
    )
}

let get_detail_info = function(item, product_info, cb){
    let detail_url = 'https://guangdiu.com/api/showdetail.php?v=21&id=' + item.id
    superagent.get(detail_url)
    .set('Host', 'guangdiu.com')
    .set('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    .set('User-Agent', 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114')
    .end(function(err, detail_res){
        let $ = cheerio.load(detail_res.text)
        console.log(item.fromsite)

        //处理京东商品详情信息
        if(item.mall == '京东商城'){
            if($('.goodcms_style3_card_one', '.goodccdt').html() != null){
                //价格网 商品列表
                cb(null, 'success')
                return
            }
            let new_description = ''
            if($('p', '.mdabstract').text() != ''){
                $('p', '.mdabstract').each(function(i, elem){
                    new_description = new_description + $(this).text() + '\n'
                })
            }
            if($('#zdm-detail', '.mdabstract').text() != ''){
                $('#zdm-detail', '.mdabstract').each(function(i, elem){
                    new_description = new_description + $(this).text() + '\n'
                })
            }
            if($('.desc-con', '.mdabstract').text() != ''){
                $('.desc-con', '.mdbcontent').each(function(i, elem){
                    if($(this).text().match(/\S+/) != null){
                        new_description = new_description + $(this).text() + '\n'
                    }else{

                    }
                })
            }
            product_info.description = new_description
            product_info.headinglink = 'https://guangdiu.com' + $('.mdtitle', '.mdmain').attr('href').match(/..(\S+)/)[1]
            get_jd_item_link(product_info.headinglink, function(item_link){
                if(item_link == ''){
                    cb(null ,'success')
                }else{
                    get_jd_promotion_link.get_jd_promotion_link(item_link, function(promotion_link){
                        if(promotion_link != ''){
                            product_info.headinglink = promotion_link
                        }
                        store_data(product_info, cb)
                    })
                }
            })
        }else if(item.mall == '天猫'){  
            //处理淘宝商品详情信息
            if($('.desc-con', '.mdbcontent').text() != ''){
                //没得比
                let new_description = ''
                $('.desc-con', '.mdbcontent').each(function(i, elem){
                    if($(this).text().match(/\S+/) != null){
                        new_description = new_description + $(this).text() + '\n'
                    }else{

                    }
                })
                product_info.description = new_description
                
                if($('a', '.mdbcontent').text() != ''){//https://guangdiu.com/api/showdetail.php?v=21&id=4813359
                    let pid_regx = /(\S+pid=)\S+/
                    if($('a', '.mdbcontent').attr('href').match(pid_regx) == null){
                        cb(null, 'success')//是加密链接形式
                        return
                    }else{
                        let new_link = $('a', '.mdbcontent').attr('href').match(pid_regx)[1] + tb_pid
                        product_info.headinglink = new_link
                    }
                }else{//https://guangdiu.com/api/showdetail.php?v=21&id=4813000
                    product_info.headinglink = $('.mdtitle', '.mdmain').attr('href') + '&pid=' + tb_pid
                }
            }else if($('.huihuidetail', '.mdabstract').text() != ''){
                //惠惠网 https://guangdiu.com/api/showdetail.php?v=21&id=4817779
                product_info.description = $('.editer-content', '.huihuidetail').text()
                if($('a', '.editer-content').text() != ''){
                    product_info.headinglink = $('a', '.editer-content').attr('href') + '&pid=' + tb_pid
                }else{
                    product_info.headinglink = $('.mdtitle', '.mdmain').attr('href') + '&pid=' + tb_pid
                }
            }else if($('#htyhdetail', '.mdabstract').text() != ''){
                //海淘优惠 https://guangdiu.com/api/showdetail.php?v=21&id=4817774
                product_info.description = $('#htyhdetail', '.mdabstract').text()
                if($('a', '#htyhdetail').text() != ''){
                    let pid_regx = /(\S+pid=)\S+/
                    if($('a', '#htyhdetail').attr('href').match(pid_regx) != null){
                        let new_link = $('a', '#htyhdetail').attr('href').match(pid_regx)[1] + tb_pid
                        product_info.headinglink = new_link
                    }
                    //要考虑加密链接
                }else{
                    product_info.headinglink = $('.mdtitle', '.mdmain').attr('href') + '&pid=' + tb_pid
                }
            }else if($('.zdm-tui-list').text() != ''){
                //今日值得买 商品列表
                cb(null, 'success')
                return
            }else if($('.mdabstract').text() != ''){
                //价格网-天猫 https://guangdiu.com/api/showdetail.php?v=21&id=4813337
                //价格网 4813350 惠喵 4813964
                //买手党 https://guangdiu.com/api/showdetail.php?v=21&id=4813990
                //买手党 4813989 买个便宜货 4813980 
                let new_description = ''
                $('p', '.mdabstract').each(function(i, elem){
                    if($(this).text() != ''){
                        new_description = new_description + $(this).text() + '\n'
                    }else if(new_description == ''){
                        return
                    }else{
                        return false
                    }
                })
                product_info.description = new_description

                let pid_regx_1 = /(\S+pid=)\S+/
                let pid_regx_2 = /\S+(detail.tmall.com)\S+/
                let new_link = ''
                $('a', '.mdabstract').each(function(i, elem){
                    if($(this).attr('href').match(pid_regx_1) != null){
                        new_link = $(this).attr('href').match(pid_regx_1)[1] + tb_pid
                        return false
                    }else if($(this).attr('href').match(pid_regx_2) != null){
                        new_link = $(this).attr('href') + '&pid=' + tb_pid
                    }else{

                    }
                })
                if(new_link == ''){
                    new_link = $('.mdtitle', '.mdmain').attr('href') + '&pid=' + tb_pid
                }
                product_info.headinglink = new_link

                let coupon_regx = /\S+shop.m.taobao.com\S+/
                let coupontext = ''
                let couponlink = ''
                $('a', '.mdabstract').each(function(i, elem){
                    if($(this).attr('href').match(coupon_regx) != null){
                        coupontext = '[点击领券]' + $(this).text()
                        couponlink = $(this).attr('href')
                        return false
                    }else{

                    }
                })
                product_info.coupontext = coupontext
                product_info.couponlink = couponlink
                console.log('-------')
            }

            //有问题的：今日值得买 https://guangdiu.com/api/showdetail.php?v=21&id=4818727
            store_data(product_info, cb)
        }
    })
}

function get_jd_item_link(headinglink, after_get_jd_item_link){
    console.log(headinglink)
    superagent.get(headinglink)
    .end(function(err, res){
        // console.log(res.text)
        if(res.text.match(new RegExp(/%2F(\d+)\.html/)) == null){
            after_get_jd_item_link('')
        }else{
            after_get_jd_item_link(res.text.match(new RegExp(/%2F(\d+)\.html/))[1])
        }
    })
    /*
    http://cps.yaoqing.com/yqtrack.php?site_id=94527&aid=2352&euid=gd&t=https%3A%2F%2Fitem.jd.com%2F1269268.html
    https://union-click.jd.com/jdc?e=&p=AyIHZRprFQEVD1YcUiVGTV8LRGtMR1dGFxBFC1pXUwkEAEAdQFkJBVsWBRoEUhJETEdOWmVjGGBgUldRAThRUnNuF2cvcH0bbFFdVxlsEQZVHUcVAQ4HUQpbFgkSDl4YUyUCEw9SHV8QBhQGZRtaFAMTAl0ZUx0yEg5QGVsWAxI3VRprQ2wSBlQaWhQGFgNQK1olAhYHVxlbFQYTD1cYUyUFIlcARwNKbBEGVRxrJTIiN2UaaxQ%3D&t=W1dCFFlQCxxKQgFHRE5XDVULR0UVARUPVhxSCltXWwg%3D
    https://union-click.jd.com/jda?e=&p=AyIHZRprFQEVD1YcUiVGTV8LRGtMR1dGFxBFC1pXUwkEAEAdQFkJBVsWBRoEUhJETEdOWmVjGGBgUldRAThRUnNuF2cvcH0bbFFdVxlsEQZVHUcVAQ4HUQpbFgkSDl4YUyUCEw9SHV8QBhQGZRtaFAMTAl0ZUx0yEg5QGVsWAxI3VRprQ2wSBlQaWhQGFgNQK1olAhYHVxlbFQYTD1cYUyUFIlcARwNKbBEGVRxrJTIiN2UaaxQ%3D&t=W1dCFFlQCxxKQgFHRE5XDVULR0UVARUPVhxSCltXWwg%3D&a=fCg9UgoiAwwHO1BcXkQYFFlgd3t3e1dbQ10zVRBSUll%2bAQAPDSwjLw%3d%3d&refer=norefer
    https://re.jd.com/cps/item/1269268.html?cu=true&utm_source=kong&utm_medium=tuiguang&utm_campaign=t_1000049399_18431201&utm_term=db9cbe23cbbf4569a2e17af55dc74d15&abt=3
    */
}

function store_data(product_info, cb){
    console.log(product_info)
    let query_sql = 'select * from products where guangdiuid = ' + product_info.guangdiuid
    connection.query(query_sql, function(err, results, fields){
        if(err){
            console.log('--------------')
            cb(null, 'success')
        }else if(results == ''){
            let insert_sql = 'insert into products set ?'
            connection.query(insert_sql, product_info, function(err, results, fields){
                console.log('--------------')
                cb(null, 'success')
            })
        }else{
            continue_spider = false
            console.log('--------------')
            cb(null, 'success')
        }
    })
}

//从网站获取数据
let spider_hot_list = function(){
    let page_num = 11
    let spider_time = new Date().getTime()
    let index = 0
    let hot_index = 0
    superagent.get('https://guangdiu.com/hots.php?p=' + page_num)
    .set('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8')
    .set('Accept-Encoding', 'gzip, deflate, br')
    .set('Host', 'guangdiu.com')
    .set('Referer', 'https://guangdiu.com/hots.php')
    .set('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36')
    .end(function(err, res){
        let $ = cheerio.load(res.text)
        let i = -1
        let total_num = $('.gooditem', '.zkcontent').length
        async.whilst(
            function(){
                i++
                return i < total_num
            },
            function(whilst_cb){
                let link = $('.goodname', '.zkcontent').eq(i).attr('href')
                let guangdiuid = link.match(new RegExp(/id=(\d+)/))[1]
                let select_sql = 'select * from products where guangdiuid = ' + guangdiuid
                console.log(guangdiuid)
                connection.query(select_sql, function(err, results, fields){
                    if(err || results == ''){
                        whilst_cb(null, 'success')
                    }else{
                        let update_sql = 'update products set ? where guangdiuid = ' + guangdiuid
                        index++
                        hot_index = spider_time - index
                        connection.query(update_sql, {hot_index: hot_index}, function(err, result, field){
                            console.log(err + '--' + result + '--')
                            whilst_cb(null, 'success')
                        })
                    }
                })
            },
            function(err, result){
                console.log(new Date().toLocaleString() + '抓取最热数据完毕')
                spider_nine_list()
            }
        )
    })
}
exports.spider_hot_list = spider_hot_list

let spider_nine_list = function(){
    let continue_store = true
    superagent.get('https://guangdiu.com/cheaps.php')
    .set('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8')
    .set('Accept-Encoding', 'gzip, deflate, br')
    .set('Accept-Language', 'zh-CN,zh;q=0.9')
    .set('Host', 'guangdiu.com')
    .set('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36')
    .end(function(err, res){
        let $ = cheerio.load(res.text)
        let i = -1
        async.whilst(
            function(){
                i++
                return i < $('.cheapitem', '.cheapcontent').length
            },
            function(whilst_cb){
                let pricetext = $('.cheapitem', '.cheapcontent').eq(i).find('.cheappriceword').text()
                let price = pricetext.match(new RegExp(/\d+\.*\d*/))[0] + ''
                let product_info = {
                    title: $('.cheapitem', '.cheapcontent').eq(i).find('.cheaptitleword').text(),
                    pubtime: new Date().getTime() - Math.round(Math.random() * 1000 * 60 * 30),
                    image: $('.cheapitem', '.cheapcontent').eq(i).find('.cheapimg').attr('src'),
                    coupontext: $('.cheapitem', '.cheapcontent').eq(i).find('.cheapgotocoupon').text(),
                    price: price
                }
                let link = $('.cheapitem', '.cheapcontent').eq(i).find('.cheapgotobuy').attr('href')
                if(link.match(new RegExp(/id=(\d+)/)) != null){
                    item_id = link.match(new RegExp(/id=(\d+)/))[1]
                    get_tb_link(item_id, function(tb_link, quan_price){
                        product_info.headinglink = tb_link
                        product_info.coupontext = '领' + quan_price + '元券'
                        store_nine_data(product_info, continue_store, whilst_cb)
                    })
                }else if(link.match(new RegExp(/(\S+)(coupon\/edetail)(\S+)/)) != null){
                    let edetail_match_outcome = link.match(new RegExp(/(\S+)(coupon\/edetail)(\S+)/))
                    superagent.get(edetail_match_outcome[1] + 'cp/coupon' + edetail_match_outcome[3])
                    .end(function(err, res){
                        if(err){

                        }else{
                            item_id = JSON.parse(res.text).result.item.itemId
                            get_tb_link(item_id, function(tb_link){
                                product_info.headinglink = tb_link
                                store_nine_data(product_info, continue_store, whilst_cb)
                            })
                        }
                    })
                }
            },
            function(err, result){
                console.log(new Date().toLocaleString() + '抓取9.9数据完毕')
                connection.end()
            }
        )
    })
}
exports.spider_nine_list = spider_nine_list

function get_tb_link(item_id, after_get_tb_item_link){
    console.log(item_id)
    superagent.post('http://www.qingtaoke.com/?r=transfer/create')
    .send({
        goods_id: item_id,
        s_click_link:'',
        quan_has:'',
        quan_price:10,
        tao_code:'',
        title:'韩范潮短袖T恤女2017夏学生百搭休闲',
        price:39,
        used_price:29,
        wenan:'纯棉面料，含棉量100%，微弹舒适，柔软亲肤，不起球、不缩水、不掉色，',
        activityId: '',
        sellerId:'1098995397',
        imageSrc:'http://imgproxy.qingtaoke.com/imgextra/i3/*t*b1*cw*fr*rp*x*x*x*xc1ap*x*x*x*x*x*x*x*x*x*x_!!0-item_pic.jpg_310x310.jpg',
        sell_num:7137,
        yongjin:30.5,
        dx: '',
        juhuasuan:1,
        pid:'mm_114783873_21908037_73486917',//买啥-py
        type:'wx',//qq
        c: '',
        s_c:''
    })
    .set('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
    .set('Cookie', 'qtkwww_=6f43f64b188233913c0c4538667e5e69bd2cbf03a%3A4%3A%7Bi%3A0%3Bs%3A6%3A%22148685%22%3Bi%3A1%3Bs%3A11%3A%2213811619374%22%3Bi%3A2%3Bi%3A2592000%3Bi%3A3%3Ba%3A0%3A%7B%7D%7D;')
    .end(function(err, res){
        console.log('get_tb_link---------------start')
        console.log(res.text)
        if(err){
            after_get_tb_item_link('')
        }else if(JSON.parse(res.text).status != 0){
            after_get_tb_item_link('')
        }else{
            after_get_tb_item_link(JSON.parse(res.text).data.url, JSON.parse(res.text).data.quan.price)
            console.log(JSON.parse(res.text).data.url)
        }
    })
}

let store_nine_data = function(product_info, continue_store, whilst_cb){
    if(continue_store == false){
        whilst_cb(null, 'success')
        return
    }
    console.log(product_info)
    let query_sql = 'select * from nine_products where title = ' + '"' + product_info.title + '"'
    connection.query(query_sql, function(err, results, fields){
        if(err){
            whilst_cb(null, 'success')
        }else if(results == ''){
            let insert_sql = 'insert into nine_products set ?'
            connection.query(insert_sql, product_info, function(err, results, fields){
                whilst_cb(null, 'success')
            })
        }else{
            continue_store = false
            whilst_cb(null, 'success')
        }
    })
}